{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "220141a1",
   "metadata": {},
   "source": [
    "# Play GaussianMABEnv-v0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "de214dc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import logging\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import scipy.stats as stats\n",
    "import gym\n",
    "import gym.spaces as spaces\n",
    "import gym.utils.seeding as seeding\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ec533c5",
   "metadata": {},
   "source": [
    "### Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "46bf842b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BernoulliMABEnv(gym.Env):\n",
    "    \"\"\" Multi-Armed Bandit (MAB) with Gaussian rewards \"\"\"\n",
    "\n",
    "    def __init__(self, n=10, means=None):\n",
    "        super(BernoulliMABEnv, self).__init__()\n",
    "        self.observation_space = spaces.Box(low=0, high=0, shape=(0,), dtype=np.float)\n",
    "        self.action_space = spaces.Discrete(n)\n",
    "        self.seed(0)\n",
    "        self.means = means or self.np_random.randn(n)\n",
    "\n",
    "    def seed(self, seed=None):\n",
    "        self.np_random, seed = seeding.np_random(seed)\n",
    "        return [seed,]\n",
    "\n",
    "    def reset(self):\n",
    "        return np.empty(0, dtype=np.float)\n",
    "\n",
    "    def step(self, action):\n",
    "        mean = self.means[action]\n",
    "        reward = self.np_random.normal(mean, 1)\n",
    "        observation = np.empty(0, dtype=np.float)\n",
    "        return observation, reward, True, {}\n",
    "\n",
    "\n",
    "from gym.envs.registration import register\n",
    "register(\n",
    "        id='GaussianMABEnv-v0',\n",
    "        entry_point=BernoulliMABEnv,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fec9df7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:00:00 [INFO] action_space: Discrete(10)\n",
      "00:00:00 [INFO] np_random: RandomState(MT19937)\n",
      "00:00:00 [INFO] means: [-1.41414702  0.89361907  0.30147067 -0.69240736  1.61374064 -1.02064936\n",
      "  0.04337526 -0.70744904  2.20136056 -0.62931658]\n",
      "00:00:00 [INFO] spec: EnvSpec(GaussianMABEnv-v0)\n",
      "00:00:00 [INFO] id: GaussianMABEnv-v0\n",
      "00:00:00 [INFO] entry_point: <class '__main__.BernoulliMABEnv'>\n",
      "00:00:00 [INFO] reward_threshold: None\n",
      "00:00:00 [INFO] nondeterministic: False\n",
      "00:00:00 [INFO] max_episode_steps: None\n",
      "00:00:00 [INFO] _kwargs: {}\n",
      "00:00:00 [INFO] _env_name: GaussianMABEnv\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('GaussianMABEnv-v0')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    if key == \"observation_space\":\n",
    "        continue\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2135c5dc",
   "metadata": {},
   "source": [
    "### Agent"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5883b966",
   "metadata": {},
   "source": [
    "$\\epsilon$-greedy Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "93bdd23c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class EpsilonGreedyAgent:\n",
    "    def __init__(self, env):\n",
    "        self.epsilon = 0.1\n",
    "        self.action_n = env.action_space.n\n",
    "        self.counts = np.zeros(self.action_n, dtype=np.float)\n",
    "        self.qs = np.zeros(self.action_n, dtype=np.float)\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if np.random.rand() < self.epsilon:\n",
    "            action = np.random.randint(self.action_n)\n",
    "        else:\n",
    "            action = self.qs.argmax()\n",
    "        if self.mode == 'train':\n",
    "            if done:\n",
    "                self.reward = reward # save reward\n",
    "            else:\n",
    "                self.action = action # save action\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        if self.mode == 'train':\n",
    "            self.counts[self.action] += 1\n",
    "            self.qs[self.action] += (self.reward - self.qs[self.action]) / self.counts[self.action]\n",
    "\n",
    "\n",
    "agent = EpsilonGreedyAgent(env)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a66bd73d",
   "metadata": {},
   "source": [
    "UCB1 Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ae202f52",
   "metadata": {},
   "outputs": [],
   "source": [
    "class UCB1Agent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.counts = np.zeros(self.action_n, dtype=np.float)\n",
    "        self.qs = np.zeros(self.action_n, dtype=np.float)\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        total_count = max(self.counts.sum(), 1) # lower bounded by 1\n",
    "        sqrts = np.sqrt(2 * np.log(total_count) / self.counts.clip(min=0.01))\n",
    "        ucbs = self.qs + sqrts\n",
    "        action = ucbs.argmax()\n",
    "        if self.mode == 'train':\n",
    "            if done:\n",
    "                self.reward = reward # save reward\n",
    "            else:\n",
    "                self.action = action # save action\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        if self.mode == 'train':\n",
    "            self.counts[self.action] += 1\n",
    "            self.qs[self.action] += (self.reward - self.qs[self.action]) / \\\n",
    "                    self.counts[self.action]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e2a2f30",
   "metadata": {},
   "source": [
    "Bayesian UCB Agent\n",
    "\n",
    "(Use Gaussian distribution)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f54b380c",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BayesianUCBAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.means = np.zeros(self.action_n, dtype=np.float)\n",
    "        self.stds = np.ones(self.action_n, dtype=np.float)\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        ucbs = self.means + 3 * self.stds\n",
    "        action = ucbs.argmax()\n",
    "        if self.mode == 'train':\n",
    "            if done:\n",
    "                self.reward = reward # save reward\n",
    "            else:\n",
    "                self.action = action # save action\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        if self.mode == 'train':\n",
    "            old_var_recip = self.stds[self.action] ** -2\n",
    "            old_natural_param_0 = self.means[self.action] * old_var_recip\n",
    "            self.means[self.action] = (old_natural_param_0 + self.reward) / \\\n",
    "                    (old_natural_param_0 + 1.)\n",
    "            self.stds[self.action] = 1. / np.sqrt(old_var_recip + 1.)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f7a4c98",
   "metadata": {},
   "source": [
    "Thompson Sampling Agent\n",
    "\n",
    "(Use Gaussian distribution)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e4acb1d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ThompsonSamplingAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.means = np.zeros(self.action_n, dtype=np.float)\n",
    "        self.stds = np.ones(self.action_n, dtype=np.float)\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        samples = [np.random.normal(mean, std) for mean, std in\n",
    "                zip(self.means, self.stds)]\n",
    "        action = np.argmax(samples)\n",
    "        if self.mode == 'train':\n",
    "            if done:\n",
    "                self.reward = reward # save reward\n",
    "            else:\n",
    "                self.action = action # save action\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        if self.mode == 'train':\n",
    "            old_var_recip = self.stds[self.action] ** -2\n",
    "            old_natural_param_0 = self.means[self.action] * old_var_recip\n",
    "            self.means[self.action] = (old_natural_param_0 + self.reward) / \\\n",
    "                    (old_natural_param_0 + 1.)\n",
    "            self.stds[self.action] = 1. / np.sqrt(old_var_recip + 1.)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99948a5d",
   "metadata": {},
   "source": [
    "### Online Interaction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "82f79e37",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:00:00 [INFO] trial 0: average episode reward = 2.34 ± 1.12, regret = 126.96\n",
      "00:00:00 [INFO] trial 1: average episode reward = 2.15 ± 0.83, regret = 47.71\n",
      "00:00:00 [INFO] trial 2: average episode reward = 2.21 ± 0.96, regret = 78.57\n",
      "00:00:00 [INFO] trial 3: average episode reward = 2.14 ± 0.97, regret = 37.62\n",
      "00:00:01 [INFO] trial 4: average episode reward = 2.30 ± 1.02, regret = 62.78\n",
      "00:00:01 [INFO] trial 5: average episode reward = 2.42 ± 1.04, regret = 87.73\n",
      "00:00:01 [INFO] trial 6: average episode reward = 2.14 ± 1.04, regret = 9.11\n",
      "00:00:01 [INFO] trial 7: average episode reward = 2.32 ± 1.08, regret = 56.97\n",
      "00:00:01 [INFO] trial 8: average episode reward = 2.27 ± 0.87, regret = 81.09\n",
      "00:00:01 [INFO] trial 9: average episode reward = 2.24 ± 0.95, regret = 86.85\n",
      "00:00:02 [INFO] trial 10: average episode reward = 2.02 ± 1.10, regret = 88.45\n",
      "00:00:02 [INFO] trial 11: average episode reward = 2.15 ± 1.07, regret = 66.57\n",
      "00:00:02 [INFO] trial 12: average episode reward = 2.20 ± 0.94, regret = 17.62\n",
      "00:00:02 [INFO] trial 13: average episode reward = 2.20 ± 1.08, regret = 141.68\n",
      "00:00:02 [INFO] trial 14: average episode reward = 2.13 ± 0.97, regret = 5.85\n",
      "00:00:02 [INFO] trial 15: average episode reward = 2.16 ± 1.08, regret = 108.93\n",
      "00:00:03 [INFO] trial 16: average episode reward = 2.21 ± 1.07, regret = 113.79\n",
      "00:00:03 [INFO] trial 17: average episode reward = 2.13 ± 0.95, regret = 76.50\n",
      "00:00:03 [INFO] trial 18: average episode reward = 2.24 ± 1.10, regret = 52.41\n",
      "00:00:03 [INFO] trial 19: average episode reward = 2.28 ± 0.88, regret = 72.12\n",
      "00:00:03 [INFO] trial 20: average episode reward = 2.25 ± 0.87, regret = 121.23\n",
      "00:00:03 [INFO] trial 21: average episode reward = 2.24 ± 0.88, regret = 117.41\n",
      "00:00:04 [INFO] trial 22: average episode reward = 2.25 ± 1.02, regret = 90.36\n",
      "00:00:04 [INFO] trial 23: average episode reward = 2.22 ± 0.98, regret = 96.04\n",
      "00:00:04 [INFO] trial 24: average episode reward = 2.10 ± 0.94, regret = 87.79\n",
      "00:00:04 [INFO] trial 25: average episode reward = 2.06 ± 0.99, regret = 61.21\n",
      "00:00:04 [INFO] trial 26: average episode reward = 2.20 ± 1.02, regret = 25.45\n",
      "00:00:04 [INFO] trial 27: average episode reward = 2.26 ± 0.90, regret = 60.06\n",
      "00:00:05 [INFO] trial 28: average episode reward = 2.26 ± 1.14, regret = 88.40\n",
      "00:00:05 [INFO] trial 29: average episode reward = 2.20 ± 1.02, regret = 15.33\n",
      "00:00:05 [INFO] trial 30: average episode reward = 2.22 ± 1.00, regret = 32.28\n",
      "00:00:05 [INFO] trial 31: average episode reward = 2.17 ± 0.84, regret = 26.52\n",
      "00:00:05 [INFO] trial 32: average episode reward = 2.24 ± 1.00, regret = 108.50\n",
      "00:00:05 [INFO] trial 33: average episode reward = 2.08 ± 1.03, regret = 81.73\n",
      "00:00:06 [INFO] trial 34: average episode reward = 2.34 ± 1.05, regret = 66.74\n",
      "00:00:06 [INFO] trial 35: average episode reward = 2.31 ± 0.99, regret = 81.22\n",
      "00:00:06 [INFO] trial 36: average episode reward = 2.31 ± 0.91, regret = 44.51\n",
      "00:00:06 [INFO] trial 37: average episode reward = 1.98 ± 0.94, regret = 68.92\n",
      "00:00:06 [INFO] trial 38: average episode reward = 2.05 ± 1.06, regret = 41.11\n",
      "00:00:07 [INFO] trial 39: average episode reward = 2.20 ± 0.95, regret = 63.57\n",
      "00:00:07 [INFO] trial 40: average episode reward = 2.31 ± 1.03, regret = 130.17\n",
      "00:00:07 [INFO] trial 41: average episode reward = 2.32 ± 1.08, regret = 81.51\n",
      "00:00:07 [INFO] trial 42: average episode reward = 2.15 ± 0.94, regret = 66.69\n",
      "00:00:07 [INFO] trial 43: average episode reward = 2.14 ± 1.00, regret = 108.56\n",
      "00:00:07 [INFO] trial 44: average episode reward = 2.14 ± 1.01, regret = 103.20\n",
      "00:00:08 [INFO] trial 45: average episode reward = 2.32 ± 0.88, regret = 86.08\n",
      "00:00:08 [INFO] trial 46: average episode reward = 2.31 ± 1.16, regret = 123.49\n",
      "00:00:08 [INFO] trial 47: average episode reward = 2.35 ± 1.02, regret = 43.66\n",
      "00:00:08 [INFO] trial 48: average episode reward = 2.38 ± 1.00, regret = 49.08\n",
      "00:00:08 [INFO] trial 49: average episode reward = 2.28 ± 1.05, regret = 79.82\n",
      "00:00:08 [INFO] trial 50: average episode reward = 2.11 ± 1.11, regret = 67.46\n",
      "00:00:09 [INFO] trial 51: average episode reward = 2.15 ± 0.97, regret = 54.54\n",
      "00:00:09 [INFO] trial 52: average episode reward = 2.29 ± 0.90, regret = -20.30\n",
      "00:00:09 [INFO] trial 53: average episode reward = 2.21 ± 0.96, regret = 57.71\n",
      "00:00:09 [INFO] trial 54: average episode reward = 2.32 ± 1.07, regret = 15.32\n",
      "00:00:09 [INFO] trial 55: average episode reward = 2.28 ± 1.04, regret = 30.03\n",
      "00:00:09 [INFO] trial 56: average episode reward = 2.32 ± 1.07, regret = 74.01\n",
      "00:00:10 [INFO] trial 57: average episode reward = 2.17 ± 1.05, regret = 78.64\n",
      "00:00:10 [INFO] trial 58: average episode reward = 2.04 ± 0.95, regret = 62.36\n",
      "00:00:10 [INFO] trial 59: average episode reward = 2.27 ± 0.94, regret = 79.83\n",
      "00:00:10 [INFO] trial 60: average episode reward = 2.15 ± 0.99, regret = 3.52\n",
      "00:00:10 [INFO] trial 61: average episode reward = 2.17 ± 0.98, regret = 95.72\n",
      "00:00:11 [INFO] trial 62: average episode reward = 2.22 ± 0.89, regret = 41.26\n",
      "00:00:11 [INFO] trial 63: average episode reward = 2.32 ± 0.94, regret = 34.11\n",
      "00:00:11 [INFO] trial 64: average episode reward = 2.40 ± 1.01, regret = 62.67\n",
      "00:00:11 [INFO] trial 65: average episode reward = 2.19 ± 0.93, regret = 102.29\n",
      "00:00:11 [INFO] trial 66: average episode reward = 2.10 ± 0.96, regret = 25.34\n",
      "00:00:11 [INFO] trial 67: average episode reward = 2.21 ± 0.99, regret = 75.28\n",
      "00:00:12 [INFO] trial 68: average episode reward = 2.15 ± 1.11, regret = 37.63\n",
      "00:00:12 [INFO] trial 69: average episode reward = 2.30 ± 1.11, regret = 87.22\n",
      "00:00:12 [INFO] trial 70: average episode reward = 2.09 ± 0.99, regret = 88.74\n",
      "00:00:12 [INFO] trial 71: average episode reward = 2.30 ± 0.98, regret = 65.33\n",
      "00:00:12 [INFO] trial 72: average episode reward = 2.15 ± 0.98, regret = 115.17\n",
      "00:00:12 [INFO] trial 73: average episode reward = 2.01 ± 1.01, regret = 62.51\n",
      "00:00:13 [INFO] trial 74: average episode reward = 2.07 ± 0.94, regret = 96.08\n",
      "00:00:13 [INFO] trial 75: average episode reward = 2.28 ± 1.05, regret = 70.02\n",
      "00:00:13 [INFO] trial 76: average episode reward = 2.29 ± 0.89, regret = 83.63\n",
      "00:00:13 [INFO] trial 77: average episode reward = 2.17 ± 1.01, regret = 56.54\n",
      "00:00:13 [INFO] trial 78: average episode reward = 2.09 ± 0.91, regret = 69.10\n",
      "00:00:13 [INFO] trial 79: average episode reward = 2.19 ± 0.88, regret = 73.64\n",
      "00:00:14 [INFO] trial 80: average episode reward = 2.25 ± 0.95, regret = 52.69\n",
      "00:00:14 [INFO] trial 81: average episode reward = 2.13 ± 0.93, regret = 122.55\n",
      "00:00:14 [INFO] trial 82: average episode reward = 2.23 ± 1.04, regret = 96.24\n",
      "00:00:14 [INFO] trial 83: average episode reward = 2.19 ± 0.94, regret = 120.24\n",
      "00:00:14 [INFO] trial 84: average episode reward = 2.38 ± 1.09, regret = 73.32\n",
      "00:00:14 [INFO] trial 85: average episode reward = 2.26 ± 1.02, regret = 72.13\n",
      "00:00:15 [INFO] trial 86: average episode reward = 2.00 ± 1.04, regret = 101.56\n",
      "00:00:15 [INFO] trial 87: average episode reward = 2.24 ± 1.09, regret = 58.79\n",
      "00:00:15 [INFO] trial 88: average episode reward = 2.02 ± 0.98, regret = 57.03\n",
      "00:00:15 [INFO] trial 89: average episode reward = 2.30 ± 1.01, regret = 17.68\n",
      "00:00:15 [INFO] trial 90: average episode reward = 2.30 ± 0.93, regret = 34.90\n",
      "00:00:16 [INFO] trial 91: average episode reward = 1.69 ± 0.85, regret = 57.10\n",
      "00:00:16 [INFO] trial 92: average episode reward = 2.21 ± 0.99, regret = 93.34\n",
      "00:00:16 [INFO] trial 93: average episode reward = 2.20 ± 1.05, regret = 113.55\n",
      "00:00:16 [INFO] trial 94: average episode reward = 2.21 ± 0.91, regret = 93.72\n",
      "00:00:16 [INFO] trial 95: average episode reward = 2.22 ± 1.06, regret = 97.84\n",
      "00:00:16 [INFO] trial 96: average episode reward = 2.39 ± 0.96, regret = 104.35\n",
      "00:00:17 [INFO] trial 97: average episode reward = 2.02 ± 0.95, regret = 95.15\n",
      "00:00:17 [INFO] trial 98: average episode reward = 2.31 ± 0.98, regret = 53.42\n",
      "00:00:17 [INFO] trial 99: average episode reward = 2.27 ± 1.02, regret = 24.76\n",
      "00:00:17 [INFO] average regret = 70.56 ± 32.08\n"
     ]
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "trial_regrets = []\n",
    "for trial in range(100):\n",
    "    # create a new agent for each trial - change agent here\n",
    "    # agent = EpsilonGreedyAgent(env)\n",
    "    agent = UCB1Agent(env)\n",
    "    # agent = BayesianUCBAgent(env)\n",
    "    # agent = ThompsonSamplingAgent(env)\n",
    "\n",
    "    # train\n",
    "    episode_rewards = []\n",
    "    for episode in range(1000):\n",
    "        episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "                max_episode_steps=env.spec.max_episode_steps, mode='train')\n",
    "        episode_rewards.append(episode_reward)\n",
    "    regrets = env.means.max() - np.array(episode_rewards)\n",
    "    trial_regret = regrets.sum()\n",
    "    trial_regrets.append(trial_regret)\n",
    "\n",
    "    # test\n",
    "    episode_rewards = []\n",
    "    for episode in range(100):\n",
    "        episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "        episode_rewards.append(episode_reward)\n",
    "    logging.info('trial %d: average episode reward = %.2f ± %.2f, regret = %.2f',\n",
    "            trial, np.mean(episode_rewards), np.std(episode_rewards),\n",
    "            trial_regret)\n",
    "\n",
    "logging.info('average regret = %.2f ± %.2f',\n",
    "        np.mean(trial_regrets), np.std(trial_regrets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a0bff45b",
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
